const _ = require('underscore')
const promisePipe = require("promisepipe");
const request = require('request');
const fs = require('fs');
const compressing = require('compressing');

const targetDir = __dirname + '/data'
const urlHeader = 'http://chengjinglearn.qiniudn.com/TGraphDemo/'
const fileList = [
    "20101104.tar.gz",
    "20101105.tar.gz",
    "20101106.tar.gz",
    "20101107.tar.gz",
    "20101108.tar.gz"
]

function listDir(dir) {
    if (dir.slice(-1) != '/') dir += '/'
    var result = []
    var fileList = fs.readdirSync(dir, 'utf-8');
    for (var i in fileList) {
        var stat = fs.lstatSync(dir + fileList[i]);
        // 是目录，需要继续
        if (stat.isDirectory()) {
            result = result.concat(listDir(dir + fileList[i] + '/'))
        } else if (fileList[i].toUpperCase().endsWith('.CSV')) {
            result.push(dir + fileList[i])
            /* console.log(dir + fileList[i]) */
        }
    }
    return result
}

async function download(count) {
    if (!fs.existsSync(__dirname + "/data")) {
        fs.mkdirSync(__dirname + "/data")
    }
    await Promise.all(fileList.slice(0, count).map(async function (fileName) {
        await promisePipe(request.get(urlHeader + fileName), fs.createWriteStream(__dirname + "/data/" + fileName))
        console.trace('download ' + fileName + " done.")
        await compressing.tgz.uncompress(__dirname + "/data/" + fileName, __dirname + "/data")
        console.trace('unzip ' + fileName + " done.")
    }))
}

var minTime = Infinity
var maxTime = 0

async function readFile(count) {
    var csvFiles = listDir(__dirname + "/data/")
    if (csvFiles.length < count) {
        await download(parseInt(Math.ceil(count / 288)))
        csvFiles = listDir(__dirname + "/data/")
        console.debug('download done.')
    }
    return _.flatten(csvFiles.slice(0, count).map(function (filePath) {
        var timeStr = filePath.match(/TJamData_(.+)\.csv/)
        var time
        if (timeStr) {
            var c = timeStr[1]
            var timestamp = Date.UTC(c.slice(0, 4), c.slice(4, 6) - 1, c.slice(6, 8), c.slice(8, 10), c.slice(10, 12)) + 8 * 3600 * 1000
            time = timestamp / 1000
            if (time > maxTime) maxTime = time
            if (time < minTime) minTime = time
        } else {
            throw 'should not happen: match fileName failed.'
        }
        var data = fs.readFileSync(filePath);
        var lines = data.toString().split('\n')
        lines.shift()
        lines.pop()
        return lines.map(function (line) {
            var fields = line.split(',')
            return [fields[1], fields[2], time, fields[6], fields[7], fields[8], fields[9]]
        })
        /* .filter(function(line){
            return line!=''
        }) */
    }), true)
}

var dataStartPoint = {
    travelTime: {},
    status: {},
    carCount: {},
    segCount: {}
}

var dataTimeInterval = {
    travelTime: {},
    status: {},
    carCount: {},
    segCount: {}
}

function rawToStartPoint(data) {
    _.each(data, function (item) {
        var roadId = item[0] + ':' + item[1]
        var time = item[2]
        var p = {
            travelTime: item[3],
            status: item[4],
            carCount: item[5],
            segCount: item[6]
        }
        _.each(p, function (val, key) {
            if (_.has(dataStartPoint[key], roadId)) {
                dataStartPoint[key][roadId].push([time, val])
            } else {
                dataStartPoint[key][roadId] = [[time, val]]
            }
        })
    })
}

function toTimeInterval() {
    _.each(dataStartPoint, function (val, key) {
        _.each(val, function (lst, roadId) {
            var interval = []
            var i = 1, j = 0
            for (i = 1, j = 0; i < lst.length; i++) {
                var preVal = lst[j][1]
                var curVal = lst[i][1]
                if (curVal != preVal) {
                    interval.push([lst[j][0], lst[i][0] - 1, preVal])
                    j = i
                }
            }
            if (j < i && lst.length > 0) {
                interval.push([lst[j][0], maxTime, lst[j][1]])
            }
            dataTimeInterval[key][roadId] = interval
        })
    })
}

var idMap = {}

function roadIdMapping(){
    var ids = []
    _.each(dataStartPoint, function (val, key) {
        _.each(val, function (lst, roadId) {
            ids.push(roadId)
        })
    })
    ids.sort()
    var ids = _.uniq(ids, true)
    for(var i in ids){
        idMap[ids[i]] = i
    }
}

var indexEntries = {
    travelTime: {},
    status: {},
    carCount: {},
    segCount: {}
}

function singlePropertyIndexEntries(){
    _.each(dataTimeInterval, function(val, pName){
        var result = []
        _.each(val, function(pVal, roadId){
            _.each(pVal, function(item){
                result.push( [ idMap[roadId], item[0], item[1], item[2] ])
            })
        })
        indexEntries[pName] = result
    })
}

async function getIndexEntries(timeCount){
    var data = await readFile(timeCount)
    rawToStartPoint(data)
    roadIdMapping()
    toTimeInterval()
    singlePropertyIndexEntries()
    return indexEntries
}

module.exports = {
    download: download,
    readFile: readFile,
    getIndexEntries: getIndexEntries
}